﻿using System;
using System.Text;
using System.Collections.Generic;
using System.Linq;
using Microsoft.VisualStudio.TestTools.UnitTesting;

namespace Cips.DataMining.Tests
{
    using System.ServiceModel.Channels;

    /// <summary>
    /// Summary description for TokenizerTests
    /// </summary>
    [TestClass]
    public class TokenizerTests
    {
        [TestMethod]
        public void TokenizerShouldSplitWords()
        {
            const string Input = "this is a test string";

            CollectionAssert.AreEqual(
                new[] { "this", "is", "test", "string" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldSplitAtDashes()
        {
            const string Input = "this - is - one-test string";

            CollectionAssert.AreEqual(
                new[] { "this", "is", "one", "test", "string" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldHandleNumbers()
        {
            const string Input = "2 is a number";

            CollectionAssert.AreEqual(
                new[] { "<number>", "is", "number" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldHandlePunctuation()
        {
            const string Input = "It's function, 'none' [1]";

            CollectionAssert.AreEqual(
                new[] { "It's", "function", "none", "<number>" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldHandleEmails()
        {
            const string Input = "user@host.com, not user@ host but user@host.com.tr";

            CollectionAssert.AreEqual(
                new[] { "<email>", "not", "user", "host", "but", "<email>" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldSkipSingleChars()
        {
            const string Input = "only this a b c";

            CollectionAssert.AreEqual(
                new[] { "only", "this" },
                TextCleaner.Tokenize(Input).ToList());
        }

        [TestMethod]
        public void TokenizerShouldHandleUrls()
        {
            const string Input = "http://url.com/should?be#skipped&to work";

            CollectionAssert.AreEqual(
                new[] { "<url>", "work" },
                TextCleaner.Tokenize(Input).ToList());
        }
    }
}
